package com.yjjxt

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel

object Hello05Persist {
  def main(args: Array[String]): Unit = {
    //1.配置并创建对象
    val sparkContext = new SparkContext((new SparkConf().setMaster("local").setAppName("Hello05Persist" + System.currentTimeMillis())))
    //2.开始读取数据
    var lines: RDD[String] = sparkContext.textFile("src/main/resources/NASA_access_log_Aug95")
    //3.开始进行缓存
    lines = lines.persist(StorageLevel.DISK_ONLY)
    //4.开始进行计算
    val startTime = System.currentTimeMillis
    val count = lines.count
    val endTime = System.currentTimeMillis
    System.out.println("第一次共" + count + "条数据，" + "计算时间=" + (endTime - startTime))
    val cacheStartTime = System.currentTimeMillis
    val cacheResult = lines.count
    val cacheEndTime = System.currentTimeMillis
    System.out.println("第二次共" + cacheResult + "条数据，" + "计算时间=" + (cacheEndTime - cacheStartTime))
    //关闭sparkContext
    sparkContext.stop()
  }
}
